import json
from datetime import datetime, timedelta
import scrapy
from scrapy.http import Request
from spixiv.items import IllustrationItem, PageItem
from spixiv.models import db, Page


class PixivSpider(scrapy.Spider):
    """Spider for daily top illustrations on Pixiv
    """
    name = 'pixiv'
    allowed_domains = ['pixiv.net']
    url_pattern = 'http://www.pixiv.net/ranking.php?mode=daily&content=illust&format=json&date={0}&p={1}'

    keys = ['illust_id', 'title', 'url', 'user_id', 'user_name',
            'illust_upload_timestamp', 'rank', 'illust_page_count']

    def __init__(self, start=None, end=None, *args, **kwargs):
        super(PixivSpider, self).__init__(*args, **kwargs)
        if start is None or start == 'today':
            start = datetime.today().date().strftime('%Y%m%d')
        if end is None:
            end = '20000101'
        try:
            self.start_date = datetime.strptime(start, '%Y%m%d').date()
            self.end_date = datetime.strptime(end, '%Y%m%d').date()
        except ValueError:
            raise ValueError('%s or %s is not a valid date' % (start, end))
        if self.start_date < self.end_date:
            raise ValueError('%s must smaller than %s' % (self.end_date, self.start_date))

    def start_requests(self):
        db.initialize_from_uri(self.settings['DATABASE_URI'])

        db.set_autocommit(False)
        db.begin()

        dt = timedelta(1)
        date = self.start_date
        while date != self.end_date:
            # check the date
            page = Page.select().where(Page.date == date).order_by(Page.p.desc()).first()
            if page is None:
                url = self.url_pattern.format(date.strftime('%Y%m%d'), None)
            else:
                url = self.url_pattern.format(page.date.strftime('%Y%m%d'), page.p)
            yield Request(url)
            date = date - dt

    def parse(self, response):
        jsondata = json.loads(response.body)

        date = jsondata['date']
        for one in jsondata['contents']:
            item = IllustrationItem()
            item['date'] = date
            for key in self.keys:
                item[key] = one[key]
            # data type
            item['illust_id'] = int(item['illust_id'])
            item['user_id'] = int(item['user_id'])
            item['illust_upload_timestamp'] = int(item['illust_upload_timestamp'])
            item['rank'] = int(item['rank'])
            item['illust_page_count'] = int(item['illust_page_count'])
            yield item
        item = PageItem()
        item['mode'] = jsondata['mode']
        item['p'] = jsondata['page']
        item['date'] = jsondata['date']
        yield item

        db.commit()

        prev_date = jsondata['prev_date']
        next_date = jsondata['next_date']
        page = jsondata['page']
        prev = jsondata['prev']
        next = jsondata['next']
        if next is not None:
            yield Request(self.url_pattern.format(date, next))

    def closed(self, reason):
        db.commit()